package com.guiji.quartz.domain;

import cn.hutool.core.util.RandomUtil;
import cn.hutool.crypto.SecureUtil;
import cn.hutool.extra.spring.SpringUtil;
import cn.hutool.http.HttpException;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.wanghaomiao.xpath.model.JXDocument;
import com.guiji.quartz.task.DataSaveUtil;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.redisson.api.RBucket;
import org.redisson.api.RSetCache;
import org.redisson.api.RedissonClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.net.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @program: cms-vue-plus
 * @description:
 * @author: wangxiaowen
 * @create: 2021-11-16 17:35
 **/
public class Marketwatch {

	private static final Logger log = LoggerFactory.getLogger(Marketwatch.class);
	private static Set<String> tabs = new HashSet<>();

	static {
		tabs.add("investing");
		tabs.add("");
		tabs.add("market-data");
	}

	public static void mainMethod() throws Exception {
		HttpResponse response = null;
		String doc = null;
		for (String tab : tabs) {

			String idxUrl = new StringBuffer("https://www.marketwatch.com/").append(tab).toString();
			try {
				response = HttpRequest.get(idxUrl)
					.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
					.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36")
					.header("Accept-Encoding", "gzip, deflate, br")
					.header("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2")
					.header("Host", "www.marketwatch.com")
					.setConnectionTimeout(15000)
					.execute();
				List<HttpCookie> cookies = response.getCookies();
				doc = response.body();
				Set<String> resUrl = getUrl(doc, tab);
				getContent(resUrl, cookies, tab);
			} catch (Exception e) {
				log.error("https://www.marketwatch.com/ 代理失败1" + e);
				return;
			}
		}


	}

	private static void getContent(Set<String> listContent, List<HttpCookie> cookies, String tab) throws InterruptedException, MalformedURLException, URISyntaxException {
		for (String url : listContent) {
			Thread.sleep(1000);
			log.info("getContent执行到-{},listContent长度-{}", url, listContent.size());
			String doc = null;
			JXDocument jxDocument = null;
			try {
				if (null != cookies && cookies.size() > 0) {
					CookieManager ccm = new CookieManager();
					CookieStore CookieStore = ccm.getCookieStore();
					for (HttpCookie cookie : cookies) {
						URL domain = new URL("https://www.marketwatch.com");
						CookieStore.add(domain.toURI(), cookie);
					}
					HttpRequest.setCookieManager(ccm);
				}

				HttpResponse response = HttpRequest.get(url)
					.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36")
					.header("Host", "www.marketwatch.com")
					.header("Referer", "https://www.marketwatch.com/")
					.setConnectionTimeout(15000).execute();
				doc = response.body();
				if (null == doc || doc.isBlank() || doc.isEmpty()) {
					log.info("请求详情页为空");
					continue;
				}
				jxDocument = new JXDocument(doc);
			} catch (HttpException e) {
				log.info("请求详情页失败" + e);
				continue;
			}
			String title = null;
			try {
				title = (String) jxDocument.sel("//div[@class='article__masthead']/h1/text()").get(0);
			} catch (Exception e) {

				try {
					title = (String) jxDocument.sel("//h1[@class='article__headline']/text()").get(0);
				} catch (Exception e2) {
					log.error("标题错误-------------" + url + "-------------------标题错误");
					continue;
				}
				log.info("first get title failed,second time success");
			}

			if (title.contains("GSTR") || title.contains("gstr") || title.contains("retire")) {
				continue;
			}

			String content = null;
			try {
				Document document = Jsoup.parse(doc);
				content = document.getElementsByClass("article__body").get(0).toString().replaceAll("\n", "").replaceAll("iframe.*?}", "");
				content = content.replaceAll("</p>.*?\n", "\\$\\$\\$").replaceAll("\n", "").replaceAll("<div id=.*?</div>", " ").replaceAll("<(.|\n)*?>", " ");
			} catch (Exception e) {
				continue;
			}

			List<Object> newImgs = new ArrayList<>();
			String img = null;
			try {
				// https://images.mktw.net/im-302745?width=540&size=1.5005861664712778 540w, https://images.mktw.net/im-302745?width=620&size=1.5005861664712778 620w, https://images.mktw.net/im-302745?width=639&size=1.5005861664712778 639w, https://images.mktw.net/im-302745?width=700&size=1.5005861664712778 700w, https://images.mktw.net/im-302745?width=700&size=1.5005861664712778&pixel_ratio=1.5 1050w, https://images.mktw.net/im-302745?width=700&size=1.5005861664712778&pixel_ratio=2 1400w, https://images.mktw.net/im-302745?width=700&size=1.5005861664712778&pixel_ratio=3 2100w
				int imgSize = jxDocument.sel("//div[@class='image-container  responsive-media article__inset__image__image']//img/@src").size();
				if (imgSize > 0) {
					String temp = null;
					for (int k = 0; k < imgSize; k++) {
						temp = jxDocument.sel("//div[@class='image-container  responsive-media article__inset__image__image']//img/@srcset").get(k).toString();
						img = temp.strip();
						if (null != img) {
							String[] temps = img.split(" ");
							if (null != temps && temps.length > 0) {
								img = temps[0];
								newImgs.add(img);
							}

						}
					}
				}

			} catch (Exception e) {
				log.error("可能没有图片");
				img = "";
			}

			//过滤广告图片
			RedissonClient redissonClient = SpringUtil.getBean(RedissonClient.class);
			boolean flag = false;
			if (newImgs.size() != 0) {
				for (Object o : newImgs) {
					HttpResponse response = HttpRequest.get(o.toString()).execute();
					String body = response.body();
					String s = SecureUtil.md5(body);
					RSetCache<Object> pic_md5 = redissonClient.getSetCache("pic_md");
					if(pic_md5.contains(s)){
						flag=true;
						break;
					}
				}
			}
			if(flag){
				continue;
			}

			Date date = new Date();

			if (null != content && content.length() > 50000 || content.replaceAll("\\$|[\\s\\p{Zs}]", "").length() < 20 || newImgs.size() > 20) {
				continue;
			}
			String source = new StringBuffer("https://www.marketwatch.com/").append(tab).toString();
			int i = RandomUtil.randomInt(1, 4);
			String category = "Marketwatch-EnglishBusiness-" + i;
			if (tab.equals("investing")) {
				category = "marketwatch-Investing";
			}
			if (tab.equals("market-data")) {
				category = "marketwatch-Market";
			}
			try {
				DataSaveUtil.saveData(title, content.toString(), newImgs, url, date, source, category);
				log.info(Thread.currentThread().getName() + "-----------------marketwatch----------------");
			} catch (Exception e) {
				continue;
			}
		}
	}

	private static Set<String> getUrl(String content, String tab) {

		Set<String> res = new HashSet<>();
		String pattern = "https://www.marketwatch.com/story/.*?mod=home-page";
		if (tab.equals("investing")) {
			pattern = "https://www.marketwatch.com/story/.*?mod=investing";
		}
		if (tab.equals("market-data")) {
			pattern = "https://www.marketwatch.com/story/.*?mod=market-data-center";
		}
		Pattern p = Pattern.compile(pattern);
		Matcher m = p.matcher(content);
		String temp = "";
		while (m.find()) {
			temp = (String) m.group();
			if(temp.contains("retire")){
				continue;
			}
			if (null != temp) {
//				if(temp.contains("/forex")||temp.contains("/commodities")||temp.contains("/markets")||temp.contains("/stocks")){
				res.add(temp);
			}
		}
		return res;
	}

//	public static void main(String[] args) throws Exception {
//		mainMethod();
//	}
}


